{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyalink.alink import *\n",
    "useLocalEnv(1)\n",
    "\n",
    "from utils import *\n",
    "import os\n",
    "import pandas as pd\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#c_1\n",
    "\n",
    "DATA_DIR = ROOT_DIR + \"german_credit\" + os.sep\n",
    "\n",
    "TRAIN_FILE = \"train.ak\";\n",
    "TEST_FILE = \"test.ak\";\n",
    "\n",
    "CLAUSE_CREATE_FEATURES = \"(case status when 'A11' then 1 else 0 end) as status_A11,\"\\\n",
    "+ \"(case status when 'A12' then 1 else 0 end) as status_A12,\"\\\n",
    "+ \"(case status when 'A13' then 1 else 0 end) as status_A13,\"\\\n",
    "+ \"(case status when 'A14' then 1 else 0 end) as status_A14,\"\\\n",
    "+ \"duration,\"\\\n",
    "+ \"(case credit_history when 'A30' then 1 else 0 end) as credit_history_A30,\"\\\n",
    "+ \"(case credit_history when 'A31' then 1 else 0 end) as credit_history_A31,\"\\\n",
    "+ \"(case credit_history when 'A32' then 1 else 0 end) as credit_history_A32,\"\\\n",
    "+ \"(case credit_history when 'A33' then 1 else 0 end) as credit_history_A33,\"\\\n",
    "+ \"(case credit_history when 'A34' then 1 else 0 end) as credit_history_A34,\"\\\n",
    "+ \"(case purpose when 'A40' then 1 else 0 end) as purpose_A40,\"\\\n",
    "+ \"(case purpose when 'A41' then 1 else 0 end) as purpose_A41,\"\\\n",
    "+ \"(case purpose when 'A42' then 1 else 0 end) as purpose_A42,\"\\\n",
    "+ \"(case purpose when 'A43' then 1 else 0 end) as purpose_A43,\"\\\n",
    "+ \"(case purpose when 'A44' then 1 else 0 end) as purpose_A44,\"\\\n",
    "+ \"(case purpose when 'A45' then 1 else 0 end) as purpose_A45,\"\\\n",
    "+ \"(case purpose when 'A46' then 1 else 0 end) as purpose_A46,\"\\\n",
    "+ \"(case purpose when 'A47' then 1 else 0 end) as purpose_A47,\"\\\n",
    "+ \"(case purpose when 'A48' then 1 else 0 end) as purpose_A48,\"\\\n",
    "+ \"(case purpose when 'A49' then 1 else 0 end) as purpose_A49,\"\\\n",
    "+ \"(case purpose when 'A410' then 1 else 0 end) as purpose_A410,\"\\\n",
    "+ \"credit_amount,\"\\\n",
    "+ \"(case savings when 'A61' then 1 else 0 end) as savings_A61,\"\\\n",
    "+ \"(case savings when 'A62' then 1 else 0 end) as savings_A62,\"\\\n",
    "+ \"(case savings when 'A63' then 1 else 0 end) as savings_A63,\"\\\n",
    "+ \"(case savings when 'A64' then 1 else 0 end) as savings_A64,\"\\\n",
    "+ \"(case savings when 'A65' then 1 else 0 end) as savings_A65,\"\\\n",
    "+ \"(case employment when 'A71' then 1 else 0 end) as employment_A71,\"\\\n",
    "+ \"(case employment when 'A72' then 1 else 0 end) as employment_A72,\"\\\n",
    "+ \"(case employment when 'A73' then 1 else 0 end) as employment_A73,\"\\\n",
    "+ \"(case employment when 'A74' then 1 else 0 end) as employment_A74,\"\\\n",
    "+ \"(case employment when 'A75' then 1 else 0 end) as employment_A75,\"\\\n",
    "+ \"installment_rate,\"\\\n",
    "+ \"(case marriage_sex when 'A91' then 1 else 0 end) as marriage_sex_A91,\"\\\n",
    "+ \"(case marriage_sex when 'A92' then 1 else 0 end) as marriage_sex_A92,\"\\\n",
    "+ \"(case marriage_sex when 'A93' then 1 else 0 end) as marriage_sex_A93,\"\\\n",
    "+ \"(case marriage_sex when 'A94' then 1 else 0 end) as marriage_sex_A94,\"\\\n",
    "+ \"(case marriage_sex when 'A95' then 1 else 0 end) as marriage_sex_A95,\"\\\n",
    "+ \"(case debtors when 'A101' then 1 else 0 end) as debtors_A101,\"\\\n",
    "+ \"(case debtors when 'A102' then 1 else 0 end) as debtors_A102,\"\\\n",
    "+ \"(case debtors when 'A103' then 1 else 0 end) as debtors_A103,\"\\\n",
    "+ \"residence,\"\\\n",
    "+ \"(case property when 'A121' then 1 else 0 end) as property_A121,\"\\\n",
    "+ \"(case property when 'A122' then 1 else 0 end) as property_A122,\"\\\n",
    "+ \"(case property when 'A123' then 1 else 0 end) as property_A123,\"\\\n",
    "+ \"(case property when 'A124' then 1 else 0 end) as property_A124,\"\\\n",
    "+ \"age,\"\\\n",
    "+ \"(case other_plan when 'A141' then 1 else 0 end) as other_plan_A141,\"\\\n",
    "+ \"(case other_plan when 'A142' then 1 else 0 end) as other_plan_A142,\"\\\n",
    "+ \"(case other_plan when 'A143' then 1 else 0 end) as other_plan_A143,\"\\\n",
    "+ \"(case housing when 'A151' then 1 else 0 end) as housing_A151,\"\\\n",
    "+ \"(case housing when 'A152' then 1 else 0 end) as housing_A152,\"\\\n",
    "+ \"(case housing when 'A153' then 1 else 0 end) as housing_A153,\"\\\n",
    "+ \"number_credits,\"\\\n",
    "+ \"(case job when 'A171' then 1 else 0 end) as job_A171,\"\\\n",
    "+ \"(case job when 'A172' then 1 else 0 end) as job_A172,\"\\\n",
    "+ \"(case job when 'A173' then 1 else 0 end) as job_A173,\"\\\n",
    "+ \"(case job when 'A174' then 1 else 0 end) as job_A174,\"\\\n",
    "+ \"maintenance_num,\"\\\n",
    "+ \"(case telephone when 'A192' then 1 else 0 end) as telephone,\"\\\n",
    "+ \"(case foreign_worker when 'A201' then 1 else 0 end) as foreign_worker,\"\\\n",
    "+ \"class \"\n",
    "\n",
    "LABEL_COL_NAME = \"class\";\n",
    "\n",
    "VEC_COL_NAME = \"vec\";\n",
    "\n",
    "PREDICTION_COL_NAME = \"pred\";\n",
    "\n",
    "PRED_DETAIL_COL_NAME = \"predinfo\";\n",
    "\n",
    "\n",
    "train_data = AkSourceBatchOp()\\\n",
    "    .setFilePath(DATA_DIR + TRAIN_FILE)\\\n",
    "    .select(CLAUSE_CREATE_FEATURES);\n",
    "\n",
    "test_data = AkSourceBatchOp()\\\n",
    "    .setFilePath(DATA_DIR + TEST_FILE)\\\n",
    "    .select(CLAUSE_CREATE_FEATURES);\n",
    "\n",
    "new_features = train_data.getColNames()\n",
    "new_features.remove(LABEL_COL_NAME)\n",
    "\n",
    "lr = LogisticRegression()\\\n",
    "    .setFeatureCols(new_features)\\\n",
    "    .setLabelCol(LABEL_COL_NAME)\\\n",
    "    .setPredictionCol(PREDICTION_COL_NAME)\\\n",
    "    .setPredictionDetailCol(PRED_DETAIL_COL_NAME);\n",
    "\n",
    "pipeline = Pipeline().add(lr);\n",
    "\n",
    "gridSearch = GridSearchCV()\\\n",
    "    .setNumFolds(5)\\\n",
    "    .setEstimator(pipeline)\\\n",
    "    .setParamGrid(\n",
    "        ParamGrid()\\\n",
    "            .addGrid(lr, 'L_1',\n",
    "                     [0.0000001, 0.000001, 0.00001, 0.0001, 0.001, 0.01, 0.1, 1.0, 10.0])\n",
    "    )\\\n",
    "    .setTuningEvaluator(\n",
    "        BinaryClassificationTuningEvaluator()\\\n",
    "            .setLabelCol(LABEL_COL_NAME)\\\n",
    "            .setPredictionDetailCol(PRED_DETAIL_COL_NAME)\\\n",
    "            .setTuningBinaryClassMetric('AUC')\n",
    "   )\\\n",
    "   .enableLazyPrintTrainInfo();\n",
    "\n",
    "bestModel = gridSearch.fit(train_data);\n",
    "\n",
    "bestModel\\\n",
    "    .transform(test_data)\\\n",
    "    .link(\n",
    "        EvalBinaryClassBatchOp()\\\n",
    "            .setPositiveLabelValueString(\"2\")\\\n",
    "            .setLabelCol(LABEL_COL_NAME)\\\n",
    "            .setPredictionDetailCol(PRED_DETAIL_COL_NAME)\\\n",
    "            .lazyPrintMetrics(\"GridSearchCV\")\n",
    "    );\n",
    "\n",
    "BatchOperator.execute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#c_2\n",
    "DATA_DIR = ROOT_DIR + \"tmall\" + os.sep\n",
    "\n",
    "ORIGIN_FILE = \"tmall.csv\";\n",
    "\n",
    "TRAIN_SAMPLE_FILE = \"train_sample.ak\";\n",
    "\n",
    "LABEL_COL_NAME = \"label\";\n",
    "PREDICTION_COL_NAME = \"pred\";\n",
    "PRED_DETAIL_COL_NAME = \"predInfo\";\n",
    "\n",
    "sw = Stopwatch();\n",
    "sw.start();\n",
    "\n",
    "train_sample = AkSourceBatchOp().setFilePath(DATA_DIR + TRAIN_SAMPLE_FILE);\n",
    "\n",
    "test_data = AkSourceBatchOp().setFilePath(DATA_DIR + TEST_FILE);\n",
    "\n",
    "featureColNames = train_sample.getColNames()\n",
    "featureColNames.remove(LABEL_COL_NAME)\n",
    "\n",
    "gbdt = GbdtClassifier()\\\n",
    "    .setFeatureCols(featureColNames)\\\n",
    "    .setLabelCol(LABEL_COL_NAME)\\\n",
    "    .setPredictionCol(PREDICTION_COL_NAME)\\\n",
    "    .setPredictionDetailCol(PRED_DETAIL_COL_NAME);\n",
    "\n",
    "randomSearch = RandomSearchTVSplit()\\\n",
    "    .setNumIter(20)\\\n",
    "    .setTrainRatio(0.8)\\\n",
    "    .setEstimator(gbdt)\\\n",
    "    .setParamDist(\n",
    "        ParamDist()\\\n",
    "            .addDist(gbdt, 'NUM_TREES', ValueDist.randArray([50, 100]))\\\n",
    "            .addDist(gbdt, 'MAX_DEPTH', ValueDist.randInteger(4, 10))\\\n",
    "            .addDist(gbdt, 'MAX_BINS', ValueDist.randArray([64, 128, 256, 512]))\\\n",
    "            .addDist(gbdt, 'LEARNING_RATE', ValueDist.randArray([0.3, 0.1, 0.01]))\n",
    "    )\\\n",
    "    .setTuningEvaluator(\n",
    "        BinaryClassificationTuningEvaluator()\\\n",
    "            .setLabelCol(LABEL_COL_NAME)\\\n",
    "            .setPredictionDetailCol(PRED_DETAIL_COL_NAME)\\\n",
    "            .setTuningBinaryClassMetric('F1')\n",
    "    )\\\n",
    "    .enableLazyPrintTrainInfo();\n",
    "\n",
    "bestModel = randomSearch.fit(train_sample);\n",
    "\n",
    "bestModel\\\n",
    "    .transform(test_data)\\\n",
    "    .link(\n",
    "        EvalBinaryClassBatchOp()\\\n",
    "            .setPositiveLabelValueString(\"1\")\\\n",
    "            .setLabelCol(LABEL_COL_NAME)\\\n",
    "            .setPredictionDetailCol(PRED_DETAIL_COL_NAME)\\\n",
    "            .lazyPrintMetrics()\n",
    "    );\n",
    "\n",
    "BatchOperator.execute();\n",
    "\n",
    "sw.stop();\n",
    "print(sw.getElapsedTimeSpan());"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#c_3\n",
    "\n",
    "DATA_DIR = ROOT_DIR + \"iris\" + os.sep\n",
    "\n",
    "VECTOR_FILE = \"iris_vec.ak\";\n",
    "\n",
    "LABEL_COL_NAME = \"category\";\n",
    "VECTOR_COL_NAME = \"vec\";\n",
    "PREDICTION_COL_NAME = \"cluster_id\";\n",
    "\n",
    "sw = Stopwatch();\n",
    "sw.start();\n",
    "\n",
    "source = AkSourceBatchOp().setFilePath(DATA_DIR + VECTOR_FILE);\n",
    "\n",
    "kmeans = KMeans()\\\n",
    "    .setVectorCol(VECTOR_COL_NAME)\\\n",
    "    .setPredictionCol(PREDICTION_COL_NAME);\n",
    "\n",
    "cv = GridSearchCV()\\\n",
    "    .setNumFolds(4)\\\n",
    "    .setEstimator(kmeans)\\\n",
    "    .setParamGrid(\n",
    "        ParamGrid()\\\n",
    "            .addGrid(kmeans, 'K', [2, 3, 4, 5, 6])\\\n",
    "            .addGrid(kmeans, 'DISTANCE_TYPE', ['EUCLIDEAN', 'COSINE'])\n",
    "    )\\\n",
    "    .setTuningEvaluator(\n",
    "        ClusterTuningEvaluator()\\\n",
    "            .setVectorCol(VECTOR_COL_NAME)\\\n",
    "            .setPredictionCol(PREDICTION_COL_NAME)\\\n",
    "            .setLabelCol(LABEL_COL_NAME)\\\n",
    "            .setTuningClusterMetric('RI')\n",
    "    )\\\n",
    "    .enableLazyPrintTrainInfo();\n",
    "\n",
    "bestModel = cv.fit(source);\n",
    "\n",
    "bestModel\\\n",
    "    .transform(source)\\\n",
    "    .link(\n",
    "        EvalClusterBatchOp()\\\n",
    "            .setLabelCol(LABEL_COL_NAME)\\\n",
    "            .setVectorCol(VECTOR_COL_NAME)\\\n",
    "            .setPredictionCol(PREDICTION_COL_NAME)\\\n",
    "            .lazyPrintMetrics()\n",
    "    );\n",
    "\n",
    "BatchOperator.execute();\n",
    "\n",
    "sw.stop();\n",
    "print(sw.getElapsedTimeSpan());"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
